{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reinforcement Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reward for this episodes was:', 30.0)\n",
      "('Reward for this episodes was:', 15.0)\n",
      "('Reward for this episodes was:', 27.0)\n",
      "('Reward for this episodes was:', 23.0)\n",
      "('Reward for this episodes was:', 12.0)\n",
      "('Reward for this episodes was:', 21.0)\n",
      "('Reward for this episodes was:', 28.0)\n",
      "('Reward for this episodes was:', 14.0)\n",
      "('Reward for this episodes was:', 11.0)\n",
      "('Reward for this episodes was:', 17.0)\n"
     ]
    }
   ],
   "source": [
    "#a example about CartPole\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import gym\n",
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "#env.reset\n",
    "env.reset()\n",
    "random_episodes=0\n",
    "reward_sum = 0\n",
    "while random_episodes < 10:\n",
    "    env.render()\n",
    "    observation, reward, done, _=env.step(np.random.randint(0,2))\n",
    "    reward_sum+=reward\n",
    "    if done:\n",
    "        random_episodes +=1\n",
    "        print(\"Reward for this episodes was:\",reward_sum)\n",
    "        reward_sum=0\n",
    "        env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reward for this episodes was:', 20.0)\n",
      "('Reward for this episodes was:', 55.0)\n",
      "('Reward for this episodes was:', 18.0)\n",
      "('Reward for this episodes was:', 14.0)\n",
      "('Reward for this episodes was:', 13.0)\n",
      "('Reward for this episodes was:', 14.0)\n",
      "('Reward for this episodes was:', 28.0)\n",
      "('Reward for this episodes was:', 16.0)\n",
      "('Reward for this episodes was:', 23.0)\n",
      "('Reward for this episodes was:', 35.0)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Variable w1 already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n  File \"<ipython-input-1-a9f1eef2494c>\", line 29, in <module>\n    w1=tf.get_variable(\"w1\",shape=[D,H],initializer=tf.contrib.layers.xavier_initializer())\n  File \"/home/lilin/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/home/lilin/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-f1defa20701d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;31m#define the architecture of the net\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0mobservations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplaceholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mD\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"input_x\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mw1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"w1\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mD\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mH\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxavier_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0mlayer1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservations\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mw1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/lilin/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)\u001b[0m\n\u001b[1;32m   1063\u001b[0m       \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1064\u001b[0m       \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1065\u001b[0;31m       use_resource=use_resource, custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m   1066\u001b[0m get_variable_or_local_docstring = (\n\u001b[1;32m   1067\u001b[0m     \"\"\"%s\n",
      "\u001b[0;32m/home/lilin/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)\u001b[0m\n\u001b[1;32m    960\u001b[0m           \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m           \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 962\u001b[0;31m           use_resource=use_resource, custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m    963\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m   def _get_partitioned_variable(self,\n",
      "\u001b[0;32m/home/lilin/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)\u001b[0m\n\u001b[1;32m    365\u001b[0m           \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    366\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 367\u001b[0;31m           validate_shape=validate_shape, use_resource=use_resource)\n\u001b[0m\u001b[1;32m    368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    369\u001b[0m   def _get_partitioned_variable(\n",
      "\u001b[0;32m/home/lilin/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource)\u001b[0m\n\u001b[1;32m    350\u001b[0m           \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m           use_resource=use_resource)\n\u001b[0m\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/lilin/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource)\u001b[0m\n\u001b[1;32m    662\u001b[0m                          \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    663\u001b[0m                          \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 664\u001b[0;31m                              name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m    665\u001b[0m       \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    666\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Variable w1 already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n  File \"<ipython-input-1-a9f1eef2494c>\", line 29, in <module>\n    w1=tf.get_variable(\"w1\",shape=[D,H],initializer=tf.contrib.layers.xavier_initializer())\n  File \"/home/lilin/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/home/lilin/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n"
     ]
    }
   ],
   "source": [
    "#a example about CartPole\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import gym\n",
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "#env.reset\n",
    "env.reset()\n",
    "random_episodes=0\n",
    "reward_sum = 0\n",
    "while random_episodes < 10:\n",
    "    env.render()\n",
    "    observation, reward, done, _=env.step(np.random.randint(0,2))\n",
    "    reward_sum+=reward\n",
    "    if done:\n",
    "        random_episodes +=1\n",
    "        print(\"Reward for this episodes was:\",reward_sum)\n",
    "        reward_sum=0\n",
    "        env.reset()\n",
    "        \n",
    "H=50\n",
    "batch_size=25\n",
    "learning_rate=1e-1\n",
    "D=4\n",
    "gamma=0.99\n",
    "\n",
    "#define the architecture of the net\n",
    "observations=tf.placeholder(tf.float32,[None,D],name=\"input_x\")\n",
    "W1=tf.get_variable(\"W1\",shape=[D,H],initializer=tf.contrib.layers.xavier_initializer())\n",
    "layer1=tf.nn.relu(tf.matmul(observations,W1))\n",
    "\n",
    "W2=tf.get_variable(\"W2\",shape=[H,1],initializer=tf.contrib.layers.xavier_initializer())\n",
    "score=tf.matmul(layer1,W2)\n",
    "probability=tf.nn.sigmoid(score)\n",
    "\n",
    "\n",
    "#\n",
    "input_y=tf.placeholder(tf.float32,[None,1],name=\"input_y\")\n",
    "advantages=tf.placeholder(tf.float32,name=\"reward_signal\")\n",
    "loglik=tf.log(input_y*(input_y-probability)+(1-input_y)*(input_y+probability))\n",
    "loss=-tf.reduce_mean(loglik*advantages)\n",
    "\n",
    "tvars=tf.trainable_variables()\n",
    "newGrades=tf.gradients(loss,tvars)\n",
    "\n",
    "#\n",
    "adam=tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "w1Grad=tf.placeholder(tf.float32,name=\"batch_grad1\")\n",
    "w2Grad=tf.placeholder(tf.float32,name=\"batch_grad2\")\n",
    "batchGrad=[w1Grad,w2Grad]\n",
    "updateGrads=adam.apply_gradients(zip(batchGrad,tvars))\n",
    "\n",
    "def discount_rewards(r):\n",
    "    discounted_r=np.zeros_like(r)\n",
    "    running_add=0\n",
    "    for t in reversed(range(r.size)):\n",
    "        running_add=running_add*gamma+r[t]\n",
    "        discounted_r[t]=running_add\n",
    "    return discounted_r\n",
    "\n",
    "\n",
    "#define some paraments before train\n",
    "xs,ys,drs=[],[],[]\n",
    "reward_sum=0\n",
    "episode_number=1\n",
    "total_episodes=10000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    rendering=False\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    observation = env.reset()\n",
    "    \n",
    "    gradBuffer=sess.run(tvars)\n",
    "    for ix,grad in enumerate(gradBuffer):\n",
    "        gradBuffer[ix]=grad*0\n",
    "        \n",
    "    while episode_number <= total_episodes:\n",
    "        if reward_sum/batch_size>100 or rendering ==True:\n",
    "            env.render()\n",
    "            rendering=True\n",
    "        x=np.reshape(observation,[1,D])\n",
    "        \n",
    "        tfprob=sess.run(probability,feed_dict={observations:x})\n",
    "        action=1 if np.random.uniform()<tfprob else 0\n",
    "        \n",
    "        xs.append(x)\n",
    "        y=1-action\n",
    "        ys.append(y)\n",
    "        \n",
    "        observation,reward,done,info = env.step(action)\n",
    "        reward_sum+=reward\n",
    "        drs.append(reward)\n",
    "        \n",
    "        if done:\n",
    "            episode_number+=1\n",
    "            epx=np.vstack(xs)\n",
    "            epy=np.vstack(ys)\n",
    "            epr=np.vstack(drs)\n",
    "            xs,ys,drs=[],[],[]\n",
    "            \n",
    "            discounted_epr=discount_rewards(epr)\n",
    "            discounted_epr-=np.mean(discounted_epr)\n",
    "            discounted_epr/=np.std(discounted_epr)\n",
    "            \n",
    "            tGrad=sess.run(newGrads,feed_dict={observations:epx,input_y:epy,advantages:discounted_epr})\n",
    "            for ix,grad in enumerate(tGrade):\n",
    "                gradBuffer[ix]+=grad\n",
    "                \n",
    "            if episode_number %batch_size ==0:\n",
    "                sess.run(updateGrades,feed_dict={w1Grad:gradBuffer[0],w2Grad:gradBuffer[1]})\n",
    "                \n",
    "                for ix,grad in enumerate(gradBuffer):\n",
    "                    gradBuffer[ix]=grad*0\n",
    "                    \n",
    "                print('Average reward for episode %d :%f.'%(episode_number,reward_sum/batch_size))\n",
    "                \n",
    "                if reward_sum/batch_size>200:\n",
    "                    print(\"Task solved in \",episode_number,'episodes!')\n",
    "                    break\n",
    "                    \n",
    "                reward_sum=0\n",
    "                \n",
    "            observation=env.reset()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
